// ==UserScript==
// @name Extract ALEKS assessment data
// @namespace http://www.collaborative.org/tek/cedu
// @description Extract student assessment data from ALEKS. See http://www.aleks.com/.
// @include https://secure.aleks.com/*
// @require navigate ALEKS.js
// @require collect text.js
// @require parse ALEKS MA standards text.js
// @require configure.js
// ==/UserScript==

/* HOW THIS WORKS
 * To assure consistent data reports and teacher worksheets, make sure no one
 *   is reporting or generating worksheets from the ALEKS repository while you
 *   are extracting data from the ALEKS web site.
 * The procedure prompts you to make choices and notifies you of events using
 *   web browser dialog boxes. There may be an option in the dialog boxes to
 *   "prevent this page from creating additional dialogs". Don't do that.
 * Start a Firefox web browser, version 3.6 or 4. (Other versions might work.)
 * Enable the Greasemonkey extension for Firefox by clicking the little monkey.
 *   See instructions below on installing Greasemonkey.
 *   In Firefox 4, the monkey appears at the top right of your browser window.
 *   In Firefox 3.6, the monkey appears at the lower right.
 *   The monkey is bright when enabled, dim when disabled.
 * Sign in to ALEKS, http://www.aleks.com/.
 * If you want to extract data for a single class, open the class page. 
 *   Class pages should have title text in the main content area 
 *   of the page that includes the class name, e.g., "Mathematics - LV 5"
 *   and the teacher's name in parenthesis, e.g., "(Mr. Smith"). 
 * Open the Greasemonkey menu.
 *   In Firefox 4, click the drop-down button beside the little monkey.
 *   In Firefox 3.6, right click the little monkey.
 * In the Greasemonkey menu, point to "User Script Commands" to display
 *   the available commands.
 *   If no commands appear, see below to install the ALEKS data extraction
 *   script.
 *   "Extract ALEKS assessment data for a teacher"
 *   "Extract ALEKS assessment data for this class"
 * If you choose to extract all data for a single teacher, the
 *   script will display a list of teachers and prompt you to select one.
 * [Menu to select student report types and date range.]
 * Watch while the script displays student reports and invisibly extracts
 *   data and transmits it to the ALEKS repository. See below for a more
 *   technical description of what happens.
 * When the script finishes it displays a message window letting you know.
 * If the script encounters a problem it will usually let you know with
 *   a message window. Please let US know if you encounter problems.
 * If nothing happens in the browser window for more than 30 seconds,
 *   something is probably amiss. You may want to try again.
 *
 * SLIGHTLY MORE TECHNICAL DESCRIPTION
 * The Greasemonkey script runs whenever a secure (HTTPS) ALEKS web page
 *  loads. The script installs the Greasemonkey user script commands in
 *  the Greasemonkey menu.
 * The user script commands launch a data extraction process, which
 *  involves multiple page loads to navigate to the student reports.
 *  The script runs afresh each time a page is loaded, figures out
 *  where it is in the process, and continues. See more information
 *  in the "navigate ALEKS.js" file.
 * The script collects text from student report pages (see "collect text.js"),
 *  parses the text according to the expected report format 
 *  (see "parse ALEKS MA standards text.js"), and posts the results
 *  to the ALEKS repository URL (see this file, below).
 * A script invoked by the URL ("store ALEKS.pl") stores the data in a 
 *   simple SQL database, the ALEKS repository (the file "aleks.sqlite").
 * Later, an Excel script reads the repository and creates or updates
 *  math assessment worksheets.
 * Other scripts may read from the repository to 
 *   upload data to the MA Education Data Warehouse, 
 *   populate student progress reports in your student information system, 
 *   populate student progress dashboards, or for other purposes.
 *
 * INSTALLING GREASEMONKEY AND THE ALEKS DATA EXTRACTION SCRIPT.
 * https://addons.mozilla.org/en-US/firefox/addon/greasemonkey/
 * Click "Add to Firefox".
 * In Windows, open the ALEKS folder, containing the ALEKS repository
 *  and various scripts, libraries, Excel files, and things that the
 *  ALEKS data management solution uses.
 * Install the ALEKS data extraction script, named "extract ALEKS data.user.js",
 *  by dragging it from the folder and dropping it in your Firefox window.
 *  Greasemonkey will display a dialog window asking your permission to install
 *  the script.
 * Click the Install button and note a message from Greasemonkey in the Firefox
 *  status bar (at the bottom of the window) letting you know the script was
 *  successfully installed.
 */

// THE SCRIPT BEGINS HERE

// Create User Script Commands in the Greasemonkey menu.
GM_registerMenuCommand('Extract ALEKS assessment data for a teacher', extractTeacher);
GM_registerMenuCommand('Extract ALEKS assessment data for this class', extractThisClass);

// See the Javascript files included above (the @require statements)
// for more information.
// "navigate ALEKS.js" handles navigating the ALEKS web pages to display reports.
// This file, "extract ALEKS data.user.js", scrapes test results from a
// "Massachusetts standards report for a single student" page.
// It uses routines in "collect text.js" to gather the text content of the page,
// and routines in "parse ALEKS MA standards text.js" to assign text fragments
// to semantic units like student, ALEKS record, strand, standard, ALEKS item,
// and student mastery.

// Wake up, if we're in the middle of a run.
wakeUp();

var asOf;  // date/time of data extraction
var teacher;
var aleksRecord;  // description of ALEKS data record, usually an assessment
var assessDate;  // date of ALEKS assessment, if extractable from aleksRecord
var postRequest;  // nonstandard XMLHttpRequest object for posting data to ALEKS repository
var postState;  // XMLHttpRequest state while posting data to ALEKS repository - see postReport()
var postedTime;  // Time report data was posted, in milliseconds since Jan. 1, 1970
var postAllowTime = 20000;  // Number of milliseconds to allow for data post to succeed - default 20 sec
var postTimer;  // id of interval timer for postWait()
// Additional global variables are declared in the library files "navigate ALEKS.js", 
// "collect text.js", and "parse ALEKS MA standards text.js".

function parseAndPost() {
    GM_log('parseAndPost');
    // Initialize global variables in case they manage to persist between calls.
    asOf = new Date();  // now
    teacher = GM_getValue('teacher', '');
    aleksRecord = aleksRecordFromPage() || '(no ALEKS record)';
    GM_log('record ' + aleksRecord);
    assessDate = aleksRecord == '(no ALEKS record)' ? null : recordAssessDate(aleksRecord);
    GM_log('assessed ' + assessDate);
    // Traverse this document depth-first and collect text.
    collectText(document);
    GM_log(text.length + ' text fragments');
    // Splice the text into coherent lines.
    spliceText(text);
    GM_log(lines.length + ' assembled lines');
    // Identify section headers and leaders.
    scanSectionHeads(lines);
    checkSectionHeads();
    // Find report title lines.
    findTitles(lines);
    // Organize data into a structured map.
    organizeLines(lines);
    // Post the results.
    postReport();
}

function aleksRecordFromPage() {
    // Return the ALEKS record label, e.g., "11/18/2010 Assessment (Requested Assessment 1)".
    // This is only available on the student report page.
    var recordSelector = nestedId('id_record', document);
    if (recordSelector)
        return recordSelector[recordSelector.selectedIndex].text;
    else
        return null;
}

function recordAssessDate(record) {
    // Extract the assessment date from the ALEKS record label, if possible.
    // Return a Date or null.
    var assessDateExp = /(\d+)\/(\d+)\/(\d+)\s+Assessment/i;  // "m/d/y Assessment"
    var assessDateMatch = assessDateExp.exec(aleksRecord);
    if (assessDateMatch)
        return new Date(assessDateMatch[3], assessDateMatch[1] - 1, assessDateMatch[2]);
    else
        return null;
}

function nestedId(id, doc) {
    // Return an HTML element with the given id, even nested inside framesets.
    // Null if not found.
    // [Refactor this to look just at our target document.]
    var el;  // an element, or null
    var f = 0;  // index through subsidiary frames;
    el = doc.getElementById(id);  // if not null we'll coast past the loop
    while ( !el && f < doc.defaultView.frames.length ) {
        el = nestedId(id, doc.defaultView.frames[f].document);
        f++;
    }
    return el;  // could be null if not found
}

function twodigits(n) {
    // Return a string of two digits for integers between 0 and 99.
    var s = '';
    if (n < 10)
        s += '0';
    return s + String(n);
}

function dateString(dt) {
    // Format a Date as yyyy-mm-dd.
    // Return an empty string if dt is null.
    if (dt)
      return dt.getFullYear() + '-' + twodigits(dt.getMonth() + 1) + '-' + twodigits(dt.getDate());
    else
      return '';
}

function dateTimeString(dt) {
    // Format a Date as yyyy-mm-dd hh:mm:ss.
    // Return an empty string if dt is null.
    if (dt)
      return dateString(dt) + ' ' + dt.toTimeString().slice(0, 8);
    else
      return '';
}

function encodeReport() {
    var x;
    x = "asOf=" + encodeURIComponent(dateTimeString(asOf));
    x += "&teacher=" + encodeURIComponent(teacher);
    x += "&report_record=" + encodeURIComponent(aleksRecord);
    x += "&assess_date=" + encodeURIComponent(dateString(assessDate));
    x += "&report_grade=" + encodeURIComponent(grade);
    x += "&report_course=" + encodeURIComponent(course);
    x += "&report_student=" + encodeURIComponent(studentName);
    x += "&studentLastFirst=" + encodeURIComponent(GM_getValue('student', ''));
    x += "&report_json=" + encodeURIComponent(JSON.stringify(reportStructure));
    x.replace('%20', '+');  // for proper form-urlencoding of spaces
    return x;
}

function postReport() {
    // Prepare and send an HTTP POST request.
    // Wait for the request to complete, up to postAllowTime milliseconds.
    // Maintain the state of the request in postState:
    // 0 --> request exists, unsent
    // 1 --> request is opened, ready to send
    // 2 --> final response headers have been received after any redirects
    // 3 --> response body is loading
    // 4 --> done; transfer is complete or something went wrong
    var details = { method: 'POST',
        url: ALEKSRepositoryStoreURL,
        data: encodeReport(),
        headers: { "Content-Type": "application/x-www-form-urlencoded" },
        onerror: postError,
        onload: postComplete,
        onreadystatechange: postStateChange
    };
    postRequest = GM_xmlhttpRequest(details);
    postedTime = Date.now();  // record time we sent the request
    postWait();  // wait for the request to succeed
}

function postWait() {
    postTimer = window.setInterval(postCheck, 500);
    postPollUntilComplete();
}

function postPollUntilComplete() {
    // [Refactor? Combine with postCheck?]
    if (postState == 4) {
        followReportingLink();
    }
    else
        window.setTimeout(postPollUntilComplete, 500);
}

function postCheck() {
    GM_log('postCheck - ready state ' + postState + ' after ' + (Date.now() - postedTime) / 1000 + ' seconds');
    if (postState == 4) {
        // Done. Continue with next extraction.
        window.clearInterval(postTimer);
    }
    else if (Date.now() - postedTime > postAllowTime) {
        // Too much time.
        postRequest.abort();
        window.clearInterval(postTimer);
        done('The ALEKS data repository isn\'t responding', 'exception');  // halt extraction
        // [Restructure errors so all code runs within a try block.]
    }
}

function postStateChange(response) {
    GM_log('XMLHttpRequest state ' + response.readyState);
    postState = response.readyState;
}

function postError(response) {
    GM_log('Error - XMLHttpRequest response status ' + response.status + ' ' + response.statusText);
    GM_log(response.responseHeaders);
    GM_log(response.responseText);
    done('There was a problem posting data to the ALEKS repository. The response status is ' + response.status + ': ' + response.statusText +
            '  I will try to open a new tab in your browser showing any text message returned by the repository server.', 'exception');
    postState = 4;
    var url = 'data:text/html;charset=iso-8859-1,' + response.responseText;
    GM_openInTab(url);
}

function postComplete(response) {
    if (response.status == 200) {
        GM_log('Success - XMLHttpRequest response status ' + response.status + ' ' + response.statusText);
        GM_log(response.responseText);
    }
    else {
        GM_log('Complete but not OK - XMLHttpRequest response status ' + response.status + ' ' + response.statusText);
        GM_log(response.responseHeaders);
        GM_log(response.responseText);
        done('There was a problem posting data to the ALEKS repository. The response status is ' + response.status + ': ' + response.statusText +
                ' I will try to open a new tab in your browser showing any text message returned by the repository server.', 'exception');
        postState = 4;
        var url = 'data:text/html;charset=iso-8859-1,' + response.responseText;
        GM_openInTab(url);
    }
}

function postReport_debug() {
    // Create a new window and form with which to submit the report.
    // [For use during code development.]
    var formHTML =
        '<form action="http://localhost/Alekscgi/storeALEKS.pl" method="post" name="HVM_ALEKS_post_form">' +
        '<table><tr><td style="vertical-align:top">' +
        '<p><label>as of<br><input type="text" name="asOf" value="' + dateTimeString(asOf) + '" size="60"></label>' +
        '<p><label>teacher<br><input type="text" name="teacher" value="' + safe(teacher) + '" size="60"></label>' +
        '<p><label>record<br><input type="text" name="report_record" value="' + safe(aleksRecord) + '" size="60"></label>' +
        '<p><label>assessment date<br><input type="text" name="assess_date" value="' + dateString(assessDate) + '" size="60">' +
        '<p><label>grade<br><input type="text" name="report_grade" value="' + safe(grade) + '" size="60"></label>' +
        '<p><label>course<br><input type="text" name="report_course" value="' + safe(course) + '" size="60"></label>' +
        '<p><label>student<br><input type="text" name="report_student" value="' + safe(studentName) + '" size="60"></label>' +
        '<p><button>submit</button>' +
        '<td>' +
        '<label>item data<br><textarea name="report_json" rows="20" cols="60">' + 
        JSON.stringify(reportStructure) + '</textarea></label>' +
        '</table>' +
        '</form>'
        ;
    var windowHTML =
        'data:text/html;charset=utf-8,' +
        '<html><head><title>Post ALEKS data</title>' +
        '<style type="text/css">button {font-size:large}</style></head>' +
        '<body>' + formHTML + '</body>' +
        '</html>'
        ;
    var windowName = 'postWindow';
    var windowRef = window.open(windowHTML, windowName);
}

function displayText(text, windowName) {
    // Display a text string or an array of text strings on a new web page.
    // [This is a utility for use during code development.]
    if (windowName == undefined) windowName = 'displayText';  // default window name and title
    if (typeof text == 'string')
        var textHTML = '<pre>' + safe(text) + '</pre>';
    else {
        textHTML = '<pre>';
        for (var i = 0; i < text.length; i++) {
            textHTML += safe(text[i]) + '<br>';
        }
        textHTML += '</pre>';
    }
    var windowHTML =
        'data:text/html;charset=utf-8,' +
        '<html>' +
        '<head><title>' + windowName + '</title></head>' +
        '<body>' + textHTML + '</body>' +
        '</html>'
        ;
    var windowRef = window.open(windowHTML, windowName);
}

function safe(s) {
    // Return an HTML-safe string with character entities.
    s = new String(s);
    s = s.replace(/&/g, '&amp;');
    s = s.replace(/</g, '&lt;');
    s = s.replace(/>/g, '&gt;');
    s = s.replace(/"/g, '&quot;');
    return s;
}

function unSafe(s) {
    // Reverse HTML character entities.
    s = s.replace(/&amp;/g, '\\&');
    s = s.replace(/&lt;/g, '<');
    s = s.replace(/&gt;/g, '>');
    s = s.replace(/&quot;/g, '"');
    return s;
}
